{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spam Detection using LSTM neural network\n",
    "In this notebook, I will implement a recurrent neural network that perform spam detection. Using an RNN/LSTM rather than a feedforward or logistic regression is more accurate since we can include information about sequence of words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import os\n",
    "from distutils.version import LooseVersion\n",
    "import warnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.2.0\n"
     ]
    }
   ],
   "source": [
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.0'), 'Please use TensorFlow version 1.0 or newer'\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"ham\\tGo until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\nham\\tOk lar... Joking wif u oni...\\nspam\\tFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 084528100\""
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('Assets/SMSSpamCollection', 'r') as f:\n",
    "    data = f.read()\n",
    "    \n",
    "data[:300]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocessing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Remove punctuation and lowercase\n",
    "from string import punctuation\n",
    "all_text = ''.join([c for c in data if c not in punctuation])\n",
    "all_text = all_text.lower()\n",
    "\n",
    "# split label and text of each line.\n",
    "messages = all_text.split('\\n')\n",
    "messages = [x.split('\\t') for x in messages if len(x)>=1]\n",
    "[labels, texts] = np.array([list(x) for x in zip(*messages)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Example: \n",
      "Label: ham,\tText: go until jurong point crazy available only in bugis n great world la e buffet cine there got amore wat\n",
      "Label: ham,\tText: ok lar joking wif u oni\n",
      "Label: spam,\tText: free entry in 2 a wkly comp to win fa cup final tkts 21st may 2005 text fa to 87121 to receive entry questionstd txt ratetcs apply 08452810075over18s\n"
     ]
    }
   ],
   "source": [
    "print(\"Example: \")\n",
    "print(\"Label: {},\\tText: {}\".format(labels[0],texts[0]))\n",
    "print(\"Label: {},\\tText: {}\".format(labels[1],texts[1]))\n",
    "print(\"Label: {},\\tText: {}\".format(labels[2],texts[2]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "Our labels are \"spam\" or \"ham\". To use these labels in our network, we need to convert them to 0 and 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, ..., 0, 0, 0])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = np.array([1 if each == 'spam' else 0 for each in labels])\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['go',\n",
       " 'until',\n",
       " 'jurong',\n",
       " 'point',\n",
       " 'crazy',\n",
       " 'available',\n",
       " 'only',\n",
       " 'in',\n",
       " 'bugis',\n",
       " 'n',\n",
       " 'great',\n",
       " 'world',\n",
       " 'la',\n",
       " 'e',\n",
       " 'buffet',\n",
       " 'cine',\n",
       " 'there',\n",
       " 'got',\n",
       " 'amore',\n",
       " 'wat']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# All words\n",
    "all_text = ' '.join(texts)\n",
    "words = all_text.split()\n",
    "words[:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Encoding the words\n",
    "Encode the words with integers and build a dictionary that maps words to integers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "counts = Counter(words)\n",
    "vocab = sorted(counts, key=counts.get, reverse = True)\n",
    "vocab_to_int = {word: ii for ii, word in enumerate(vocab)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'todayfrom': 2907,\n",
       " 'mmmmmm': 2908,\n",
       " 'tbspersolvo': 4392,\n",
       " 'bruce': 2909,\n",
       " 'anderson': 4393,\n",
       " 'discussed': 2910,\n",
       " 'mental': 2911,\n",
       " '08000930705': 678,\n",
       " '08704439680tscs': 4394,\n",
       " 'tables': 4395,\n",
       " 'window': 2912,\n",
       " 'fool': 1841,\n",
       " 'real': 327,\n",
       " 'mushy': 4396,\n",
       " 'switch': 2913,\n",
       " 'hunlove': 4397,\n",
       " 'ffffuuuuuuu': 4398,\n",
       " 'loo': 4399,\n",
       " 'sf': 4400,\n",
       " 'drinks': 1678,\n",
       " 'home': 80,\n",
       " '09063458130': 2228,\n",
       " 'gbp5month': 4401,\n",
       " 'continue': 2915,\n",
       " 'minutes': 366,\n",
       " 'spelled': 4402,\n",
       " 'ivatte': 4403,\n",
       " 'pierre': 4404,\n",
       " 'financial': 4405,\n",
       " 'asus': 4406,\n",
       " 'c': 160,\n",
       " 'hicts': 8696,\n",
       " '4few': 6876,\n",
       " 'newest': 2336,\n",
       " 'lot': 333,\n",
       " 'subscribers': 5417,\n",
       " 'screamed': 2916,\n",
       " 'rem': 1599,\n",
       " 'rearrange': 4411,\n",
       " 'walls': 2229,\n",
       " 'dhorte': 4413,\n",
       " 'yourinclusive': 4414,\n",
       " 'stdtxtrate': 4415,\n",
       " 'confused': 2917,\n",
       " 'outreach': 4416,\n",
       " 'addicted': 1842,\n",
       " 'moms': 1042,\n",
       " 'swt': 2918,\n",
       " 'kissing': 4417,\n",
       " 'fantastic': 956,\n",
       " 'we': 37,\n",
       " 'newsby': 4536,\n",
       " 'lt3': 2251,\n",
       " '40gb': 1405,\n",
       " 'blah': 2230,\n",
       " '07801543489': 4419,\n",
       " 'wwwtxttowincouk': 2920,\n",
       " 'fix': 1600,\n",
       " 'din': 1145,\n",
       " 'weighthaha': 6800,\n",
       " 'galileo': 2922,\n",
       " 'visa': 4412,\n",
       " 'thnx': 4421,\n",
       " 'downs': 6182,\n",
       " 'detail': 6119,\n",
       " 'football': 1843,\n",
       " 'album': 2924,\n",
       " 'xxxxxxxx': 4424,\n",
       " 'aight': 354,\n",
       " '020903': 2925,\n",
       " 'booking': 2442,\n",
       " 'teethis': 4427,\n",
       " 'av': 3171,\n",
       " 'thanx4': 4429,\n",
       " 'tcsc': 7916,\n",
       " '89034': 4430,\n",
       " 'mondaynxt': 4431,\n",
       " 'party': 750,\n",
       " 'ibiza': 1865,\n",
       " 'lkpobox177hp51fl': 4433,\n",
       " 'thati': 4434,\n",
       " 'friendships': 4436,\n",
       " 'realised': 4437,\n",
       " 'yes910': 4438,\n",
       " 'results': 1601,\n",
       " 'superior': 2926,\n",
       " '9am': 4439,\n",
       " 'brothas': 2233,\n",
       " 'birla': 2265,\n",
       " 'bedreal': 4441,\n",
       " 'jan': 2234,\n",
       " '2yrs': 6118,\n",
       " 'quite': 343,\n",
       " 'boobs': 7845,\n",
       " 'xxx': 344,\n",
       " 'muz': 903,\n",
       " 'ak': 4443,\n",
       " 'walmart': 2269,\n",
       " 'phoenix': 2927,\n",
       " 'weakness': 7945,\n",
       " 'sometme': 4445,\n",
       " 'personally': 6614,\n",
       " 'sleepwellamptake': 2928,\n",
       " 'upsetits': 4446,\n",
       " 'sucks': 1250,\n",
       " '09063440451': 4448,\n",
       " 'living': 1602,\n",
       " '26th': 2929,\n",
       " 'dlf': 4449,\n",
       " 'sumfing': 4450,\n",
       " 'suggest': 2235,\n",
       " 'scrounge': 2930,\n",
       " 'ignore': 1630,\n",
       " 'greatness': 4453,\n",
       " 'couldn\\x92t': 4454,\n",
       " '09099726429': 4455,\n",
       " '09061743386': 2931,\n",
       " 'scared': 1603,\n",
       " 'anythings': 4456,\n",
       " '7pm': 4457,\n",
       " 'melnite': 7600,\n",
       " 'resume': 2932,\n",
       " 'shit': 334,\n",
       " 'itmay': 4458,\n",
       " 'could': 208,\n",
       " 'vth': 2933,\n",
       " 'hellogorgeous': 4459,\n",
       " 'question': 596,\n",
       " 'silence': 3022,\n",
       " 'blank': 1844,\n",
       " '3sentiment': 4461,\n",
       " 'breathing': 4462,\n",
       " 'truly': 1679,\n",
       " 'ki': 1406,\n",
       " 'nookii': 8777,\n",
       " 'informed': 1604,\n",
       " 'sirjii': 4464,\n",
       " 'his': 209,\n",
       " 'cutting': 2936,\n",
       " 'smidgin': 4465,\n",
       " 'lib': 2938,\n",
       " 'joanna': 2939,\n",
       " 'consistently': 5807,\n",
       " 'incorrect': 4467,\n",
       " 'thinked': 8281,\n",
       " 'surprised': 1845,\n",
       " 'uso': 4468,\n",
       " 'someonethat': 4469,\n",
       " 'granted': 4470,\n",
       " 'east': 3665,\n",
       " 'vill': 8480,\n",
       " 'confirmed': 2371,\n",
       " 'animal': 4473,\n",
       " 'hunnyjust': 6438,\n",
       " 'deals': 4475,\n",
       " 'x29': 4476,\n",
       " 'gal': 957,\n",
       " 'ph08704050406': 4904,\n",
       " 'inshah': 4477,\n",
       " 'bbs': 4908,\n",
       " 'stupidits': 2237,\n",
       " 'gibe': 4479,\n",
       " 'accent': 7301,\n",
       " 'oursso': 4481,\n",
       " 'ultimatum': 2940,\n",
       " 'bras': 4482,\n",
       " 'go': 45,\n",
       " 'brought': 1847,\n",
       " '\\x93its': 4483,\n",
       " 'breathe': 2942,\n",
       " 'suganya': 4936,\n",
       " 'wishes': 1139,\n",
       " 'theirs': 6479,\n",
       " 'trying': 279,\n",
       " 'cleared': 2238,\n",
       " 'charles': 2943,\n",
       " 'feelin': 2944,\n",
       " '300': 1605,\n",
       " '09094646631': 4485,\n",
       " 'toaday': 4486,\n",
       " 'ext': 9229,\n",
       " 'compofstuff': 4487,\n",
       " 'mistakes': 2945,\n",
       " 'during': 1140,\n",
       " 'snake': 2239,\n",
       " 'rebel': 7053,\n",
       " 'wwwwin82050couk': 2946,\n",
       " 'one': 74,\n",
       " 'miracle': 1141,\n",
       " 'medont': 4489,\n",
       " 'www07781482378com': 2947,\n",
       " 'med': 1848,\n",
       " 'assume': 1849,\n",
       " 'soooo': 4490,\n",
       " 'cloud': 4491,\n",
       " 'gdeve': 4492,\n",
       " 'beautifulmay': 6000,\n",
       " 'lover': 1043,\n",
       " '15pmin': 3604,\n",
       " 'ireneere': 4494,\n",
       " 'penny': 5377,\n",
       " 'newspapers': 4496,\n",
       " 'james': 2240,\n",
       " 'yet': 245,\n",
       " 'wap': 958,\n",
       " 'cup': 1044,\n",
       " 'intention': 6196,\n",
       " 'unlike': 4498,\n",
       " 'nearby': 4499,\n",
       " 'luck': 849,\n",
       " 'websitenow': 8742,\n",
       " '9153': 4501,\n",
       " 'westlife': 3120,\n",
       " 'sis': 646,\n",
       " 'birth': 2241,\n",
       " 'wondar': 6625,\n",
       " 'word': 345,\n",
       " 'emailed': 4504,\n",
       " 'cutefrnd': 2242,\n",
       " '0871277810710pmin': 4729,\n",
       " 'table': 3132,\n",
       " 'inches': 1850,\n",
       " 'iwana': 4508,\n",
       " 'yelling': 2949,\n",
       " 'standing': 2243,\n",
       " '£3wk': 1851,\n",
       " 'summers': 8851,\n",
       " 'someplace': 4509,\n",
       " 'mathews': 4510,\n",
       " 'nursery': 5310,\n",
       " 'stoptxtstop£150week': 4511,\n",
       " 'experiment': 4512,\n",
       " 'portions': 6661,\n",
       " 'qlynnbv': 4484,\n",
       " 'sc': 4515,\n",
       " 'el': 4516,\n",
       " 'battle': 4517,\n",
       " 'murali': 4518,\n",
       " 'got': 56,\n",
       " '3680offer': 8450,\n",
       " 'doors': 2244,\n",
       " 'tests': 2245,\n",
       " 'kodstini': 4519,\n",
       " 'purity': 1852,\n",
       " 'gold': 2951,\n",
       " 'zoom': 4520,\n",
       " 'im': 22,\n",
       " 'lemme': 1142,\n",
       " 'sips': 4521,\n",
       " '09064012103': 4523,\n",
       " 'l8r': 1407,\n",
       " 'daytime': 4524,\n",
       " 'pound': 1143,\n",
       " 'perspective': 4526,\n",
       " 'rp176781': 7701,\n",
       " 'bhayandar': 8559,\n",
       " 'ph08700435505150p': 4527,\n",
       " 'qing': 8651,\n",
       " 'favor': 2247,\n",
       " 'uncles': 1853,\n",
       " 'impression': 4528,\n",
       " 'mycallsu': 4529,\n",
       " '8lb': 4530,\n",
       " 'humans': 4531,\n",
       " 'monkeys': 2952,\n",
       " 'blessings': 2248,\n",
       " 'warming': 4532,\n",
       " 'freezing': 2249,\n",
       " 'outdoors': 4534,\n",
       " 'cum': 751,\n",
       " 'rr': 4535,\n",
       " 'jstfrnd': 2250,\n",
       " 'spoiled': 2953,\n",
       " '09065989182': 4036,\n",
       " 'support': 904,\n",
       " 'lower': 3174,\n",
       " 'chachi': 4624,\n",
       " '241004': 4418,\n",
       " 'fraction': 4537,\n",
       " 'cuddled': 4538,\n",
       " 'hurting': 3417,\n",
       " 'playi': 4541,\n",
       " 'luvs': 2956,\n",
       " 'quoting': 1606,\n",
       " 'foundurself': 4542,\n",
       " 'outages': 9438,\n",
       " 'co': 1251,\n",
       " 'todo': 4543,\n",
       " 'sweets': 2919,\n",
       " 'spaces': 4544,\n",
       " 'deeraj': 2957,\n",
       " 'fill': 1854,\n",
       " 'deck': 4545,\n",
       " 'met': 979,\n",
       " 'brilliant1thingi': 4548,\n",
       " 'std': 1045,\n",
       " 'a': 3,\n",
       " 'presence': 8775,\n",
       " 'tells': 1607,\n",
       " 'tonite': 1144,\n",
       " 'truro': 4549,\n",
       " 'coin': 2958,\n",
       " 'th': 850,\n",
       " 'open': 653,\n",
       " 'tongued': 4551,\n",
       " '21': 2959,\n",
       " 'somebody': 905,\n",
       " 'processexcellent': 4552,\n",
       " 'arngd': 9467,\n",
       " 'bblue': 4553,\n",
       " 'avent': 2231,\n",
       " 'locks': 7903,\n",
       " 'teach': 1492,\n",
       " 'jog': 4554,\n",
       " 'prince': 4555,\n",
       " 'lucyxx': 4556,\n",
       " 'lvblefrnd': 2253,\n",
       " 'commit': 4557,\n",
       " 'toopray': 2962,\n",
       " 'entey': 4558,\n",
       " 'stuffed': 4559,\n",
       " 'convincingjust': 4560,\n",
       " 'problum': 4561,\n",
       " 'teaching': 8668,\n",
       " 'pete': 959,\n",
       " 'tasts': 4562,\n",
       " '09064018838': 5440,\n",
       " 'freshers': 4563,\n",
       " 'expect': 1855,\n",
       " 'txt250com': 4564,\n",
       " 'poorly': 4565,\n",
       " 'only1more': 5322,\n",
       " '528': 8798,\n",
       " 'laid': 1856,\n",
       " 'carente': 4568,\n",
       " 'palm': 4569,\n",
       " 'connect': 1857,\n",
       " '2exit': 4570,\n",
       " 'okvarunnathu': 4571,\n",
       " 'sochte': 8789,\n",
       " 'norm': 2254,\n",
       " 'studies': 4573,\n",
       " 'tiny': 4574,\n",
       " 'a21': 4575,\n",
       " 'to': 0,\n",
       " 'spreadsheet': 4576,\n",
       " 'freaking': 4578,\n",
       " '12hrs': 647,\n",
       " 'gong': 4580,\n",
       " 'crushes': 6213,\n",
       " '100psms': 4582,\n",
       " 'named': 3232,\n",
       " 'ikno': 7982,\n",
       " 'a£150': 4584,\n",
       " 'missionary': 4585,\n",
       " 'grams': 5553,\n",
       " 'korche': 4586,\n",
       " 'dubsack': 2252,\n",
       " 'def': 1858,\n",
       " 'deadwell': 5398,\n",
       " 'split': 4588,\n",
       " 'contention': 4589,\n",
       " 'maili': 4435,\n",
       " '09058097189': 4590,\n",
       " 'minor': 2966,\n",
       " 'thrown': 4591,\n",
       " 'namemy': 4592,\n",
       " 'adewale': 8209,\n",
       " 'tagged': 7409,\n",
       " 'miltazindgi': 4593,\n",
       " '08718711108': 4594,\n",
       " 'largest': 1859,\n",
       " 'uk': 679,\n",
       " 'priest': 4595,\n",
       " '83118': 4596,\n",
       " 'standard': 1408,\n",
       " 'responcewhat': 2257,\n",
       " 'kim': 4597,\n",
       " '3510i': 1782,\n",
       " 'deluxe': 6730,\n",
       " 'dear': 122,\n",
       " '8': 680,\n",
       " '1146': 4600,\n",
       " 'sweetheart': 2969,\n",
       " 'throw': 1860,\n",
       " '3hrs': 2970,\n",
       " 'files': 2161,\n",
       " 'treated': 3160,\n",
       " 'racing': 2971,\n",
       " 'ages': 1861,\n",
       " 'hiwhat': 4602,\n",
       " 'textsweekend': 4603,\n",
       " 'beers': 4604,\n",
       " 'rich': 2258,\n",
       " 'korte': 8685,\n",
       " 'infowww100percentrealcom': 2259,\n",
       " '80062': 1608,\n",
       " 'velly': 4605,\n",
       " 'mob': 490,\n",
       " 'still': 84,\n",
       " 'staring': 2972,\n",
       " 'shitload': 3894,\n",
       " 'colleg': 4607,\n",
       " 'hiding': 4608,\n",
       " 'mutations': 4609,\n",
       " 'capital': 2973,\n",
       " 'uxxxx': 4611,\n",
       " 'only': 63,\n",
       " 'jogging': 2974,\n",
       " 'whom': 1862,\n",
       " 'travel': 1609,\n",
       " 'egbon': 4612,\n",
       " 'dentist': 4613,\n",
       " 'careful': 1863,\n",
       " 'necessary': 2975,\n",
       " '087018728737': 7852,\n",
       " 'occupy': 2976,\n",
       " 'afew': 4615,\n",
       " 'various': 2261,\n",
       " 'sfine': 7205,\n",
       " 'determine': 8055,\n",
       " 'art': 2396,\n",
       " 'loneliness': 4617,\n",
       " 'correction': 4618,\n",
       " 'helloyou': 4619,\n",
       " '08717509990': 4620,\n",
       " 'knew': 796,\n",
       " 'motivating': 4622,\n",
       " 'ears': 4623,\n",
       " '08718725756': 4625,\n",
       " 'searching': 1610,\n",
       " 'farm': 3313,\n",
       " 'qatarrakhesh': 4628,\n",
       " 'diskyou': 7116,\n",
       " '4rowdy': 4630,\n",
       " 'shattered': 4631,\n",
       " 'pshewmissing': 4632,\n",
       " 'gives': 1409,\n",
       " 'backdoor': 4633,\n",
       " 'opinion': 1182,\n",
       " 'visitors': 4634,\n",
       " 'spain': 4635,\n",
       " 'church': 1864,\n",
       " 'inclu': 4636,\n",
       " 'westonzoyland': 4637,\n",
       " '7zs': 9528,\n",
       " 'owo': 4638,\n",
       " 'opposed': 4639,\n",
       " 'disappeared': 9268,\n",
       " 'wasnt': 797,\n",
       " 'unemployed': 3442,\n",
       " 'sleepingwith': 4642,\n",
       " 'faber': 7967,\n",
       " 'cysts': 4644,\n",
       " 'dps': 4645,\n",
       " 'golf': 4646,\n",
       " 'wwwtextcompcom': 2977,\n",
       " 'appointments': 4647,\n",
       " 'advise': 2232,\n",
       " '4get': 2188,\n",
       " 'loosing': 5827,\n",
       " 'somewheresomeone': 4650,\n",
       " 'shopthe': 4651,\n",
       " 'pictures': 1664,\n",
       " 'ldn': 960,\n",
       " 'cr': 4652,\n",
       " 'goggles': 4653,\n",
       " 'area': 798,\n",
       " '220cm2': 2263,\n",
       " 'ghodbandar': 4656,\n",
       " 'conversations': 4657,\n",
       " 'november': 4658,\n",
       " 'indian': 1611,\n",
       " 'lor': 81,\n",
       " 'upd8': 3338,\n",
       " 'deepak': 3350,\n",
       " '28': 1682,\n",
       " 'fatty': 4660,\n",
       " 'echo': 4661,\n",
       " 'dawns': 6342,\n",
       " 'grand': 1966,\n",
       " 'cw25wx': 1866,\n",
       " '07046744435': 7122,\n",
       " 'designation': 9280,\n",
       " 'staff': 2981,\n",
       " 'floppy': 7303,\n",
       " 'argument': 1612,\n",
       " 'discount': 906,\n",
       " 'actual': 3513,\n",
       " 'nitros': 3901,\n",
       " 'wheel': 4667,\n",
       " 'base': 2982,\n",
       " 'yup': 280,\n",
       " 'jaya': 3370,\n",
       " 'compliments': 4668,\n",
       " 'mofo': 6103,\n",
       " 'ducking': 4670,\n",
       " 'heri': 2983,\n",
       " 'wine': 907,\n",
       " 'all': 50,\n",
       " 'moseley': 4673,\n",
       " 'ran': 1613,\n",
       " 'knowyetunde': 4440,\n",
       " 'gotany': 4676,\n",
       " 'meet': 164,\n",
       " 'themed': 4677,\n",
       " 'sonyericsson': 1614,\n",
       " 'head': 621,\n",
       " 'yes762': 4678,\n",
       " 'sachin': 2984,\n",
       " 'definite': 2985,\n",
       " 'nurungu': 2266,\n",
       " 'citylink': 4679,\n",
       " '£300': 1253,\n",
       " 'cc100pmin': 4681,\n",
       " 'youuuuu': 4682,\n",
       " 'payed': 4683,\n",
       " 'fills': 1867,\n",
       " 'brison': 5852,\n",
       " 'bout': 713,\n",
       " 'caroline': 2986,\n",
       " 'fast': 714,\n",
       " 'bathing': 1868,\n",
       " 'amy': 2987,\n",
       " 'oi': 2988,\n",
       " 'ba': 2989,\n",
       " 'freaky': 9356,\n",
       " 'old': 572,\n",
       " 'naughty': 1410,\n",
       " 'brainy': 4685,\n",
       " 'halla': 4686,\n",
       " 'w': 799,\n",
       " 'reslove': 4687,\n",
       " 'comedycant': 4688,\n",
       " 'drunk': 2267,\n",
       " 'urgran': 4690,\n",
       " 'toclaim': 1411,\n",
       " 'windy': 4691,\n",
       " 'visiting': 4692,\n",
       " 'shant': 4693,\n",
       " 'year': 246,\n",
       " 'dontcha': 4694,\n",
       " 'bottom': 2268,\n",
       " 'mquiz': 4695,\n",
       " '1013': 4696,\n",
       " 'noun': 2990,\n",
       " 'fault': 1412,\n",
       " 'collect': 440,\n",
       " 'player': 752,\n",
       " 'belligerent': 4444,\n",
       " 'messaged': 2991,\n",
       " 'walsall': 4698,\n",
       " 'repairs': 4699,\n",
       " 'improve': 2992,\n",
       " 'sian': 2993,\n",
       " 'enketa': 4700,\n",
       " 'savamob': 834,\n",
       " '6ramaduth': 4701,\n",
       " 'slovely': 4702,\n",
       " 'wwwldewcom': 4153,\n",
       " 'familymay': 4705,\n",
       " '67441233': 8333,\n",
       " '08715203677': 9559,\n",
       " 'gaze': 4708,\n",
       " 'achieve': 4709,\n",
       " 'embarassing': 4710,\n",
       " 'rightio': 4711,\n",
       " '542': 1869,\n",
       " 'whatever': 648,\n",
       " 'kalainar': 7129,\n",
       " 'help08718728876': 2994,\n",
       " 'telephone': 2995,\n",
       " 'bornplease': 4713,\n",
       " 'restocked': 6335,\n",
       " 'cheetos': 4856,\n",
       " 'rayan': 4716,\n",
       " 'accumulation': 4717,\n",
       " '9ae': 1870,\n",
       " 'welcomes': 2996,\n",
       " 'fried': 4718,\n",
       " 'boltblue': 4719,\n",
       " 'outrageous': 8825,\n",
       " 'til': 481,\n",
       " 'woulda': 3781,\n",
       " 'rahul': 4721,\n",
       " 'shaping': 8826,\n",
       " 'unconsciously': 7130,\n",
       " 'ans': 852,\n",
       " 'curious': 3000,\n",
       " 'tscs087147403231winawkage16': 3001,\n",
       " 'singing': 3002,\n",
       " 'sameso': 4725,\n",
       " 'sold': 4727,\n",
       " 'shd': 1255,\n",
       " 'beendropping': 8072,\n",
       " 'yould': 5299,\n",
       " 'dayswill': 4730,\n",
       " 'costs': 1685,\n",
       " 'walking': 1299,\n",
       " 'urgent': 194,\n",
       " 'mush': 4731,\n",
       " 'credits': 1046,\n",
       " 'anand': 4732,\n",
       " 'tea': 1414,\n",
       " 'pimples': 6193,\n",
       " 'jocks': 4734,\n",
       " 'olowoyey': 4735,\n",
       " '08002986030': 3004,\n",
       " '09094100151': 4736,\n",
       " 'hmmbad': 4737,\n",
       " 'slaaaaave': 4738,\n",
       " 'man': 265,\n",
       " 'wikipediacom': 4739,\n",
       " 'varaya': 4740,\n",
       " 'agree': 3005,\n",
       " 'chess': 4741,\n",
       " 'turn': 1871,\n",
       " 'xx': 900,\n",
       " 'idk': 1873,\n",
       " 'increase': 4742,\n",
       " 'chik': 4743,\n",
       " 'cutie': 5350,\n",
       " 'julianaland': 4744,\n",
       " 'stagwood': 7864,\n",
       " 'match': 923,\n",
       " 'reformat': 4745,\n",
       " 'brilliant': 1874,\n",
       " 'masteriastering': 4746,\n",
       " '1im': 4747,\n",
       " '23': 4748,\n",
       " 'connections': 3007,\n",
       " 'understand': 961,\n",
       " 'doug': 4751,\n",
       " 'menu': 1256,\n",
       " 'worldmay': 4752,\n",
       " 'finewhen': 4156,\n",
       " 'ukmobiledate': 4754,\n",
       " 'cool': 289,\n",
       " 'availa': 7984,\n",
       " 'loss': 2004,\n",
       " '09050000332': 4757,\n",
       " 'lk': 3008,\n",
       " 'nt': 715,\n",
       " 'national': 853,\n",
       " 'vldo': 4726,\n",
       " 'goto': 1148,\n",
       " 'rhythm': 3009,\n",
       " 'enamous': 4761,\n",
       " 'novelty': 4762,\n",
       " 'wasted': 3010,\n",
       " '10ppm': 3906,\n",
       " 'samachara': 4765,\n",
       " 'blokes': 3011,\n",
       " '4cook': 2272,\n",
       " 'meneed': 8834,\n",
       " '448712404000please': 4766,\n",
       " 'outage': 4768,\n",
       " 'lane': 3012,\n",
       " 'wellda': 4770,\n",
       " 'propose': 3528,\n",
       " 'somethings': 4773,\n",
       " 'heading': 3013,\n",
       " 'of£2000': 3908,\n",
       " 'd': 127,\n",
       " 'deleted': 1876,\n",
       " 'references': 4775,\n",
       " 'chennaibecause': 4776,\n",
       " 'meanwhile': 1877,\n",
       " '2703': 7266,\n",
       " 'helens': 4778,\n",
       " 'nor': 4779,\n",
       " 'das': 5296,\n",
       " 'jealous': 2274,\n",
       " 'sppok': 4781,\n",
       " 'speechless': 2275,\n",
       " 'sooooo': 3015,\n",
       " 'maraikara': 4546,\n",
       " 'reallyneed': 4784,\n",
       " 'marriageprogram': 4785,\n",
       " 'hmv': 962,\n",
       " 'figure': 1018,\n",
       " 'friendship': 546,\n",
       " 'yes440': 3427,\n",
       " 'fold': 4788,\n",
       " 'orangei': 4789,\n",
       " 'sunday': 1149,\n",
       " 'specs': 7962,\n",
       " 'ktv': 4791,\n",
       " 'nydc': 3667,\n",
       " 'tata': 7906,\n",
       " 'vewy': 3571,\n",
       " 'tke': 4793,\n",
       " 'jaykwon': 7986,\n",
       " '08719181513': 4158,\n",
       " 'wkend': 1878,\n",
       " 'fly': 4796,\n",
       " 'explain': 1879,\n",
       " 'aunties': 4797,\n",
       " 'having': 290,\n",
       " 'lac': 8837,\n",
       " 'alivebetter': 4799,\n",
       " 'dine': 4801,\n",
       " 'freak': 2023,\n",
       " '50p': 2277,\n",
       " 'tuition': 1880,\n",
       " 'feelingood': 4803,\n",
       " '2nd': 399,\n",
       " 'say': 140,\n",
       " 'tomorrow': 143,\n",
       " 'coulda': 9239,\n",
       " '14thmarch': 8079,\n",
       " 'downon': 6808,\n",
       " 'don': 908,\n",
       " 'privacy': 3018,\n",
       " 'wo': 3019,\n",
       " 'gnarls': 4805,\n",
       " 'cust': 1881,\n",
       " 'sao': 4806,\n",
       " 'studying': 1047,\n",
       " 'zhong': 6248,\n",
       " 'guaranteed': 247,\n",
       " 'youany': 4807,\n",
       " 'crore': 3021,\n",
       " 'informedrgdsrakheshkerala': 4808,\n",
       " 'knees': 3428,\n",
       " 'events': 4810,\n",
       " 'box326': 3378,\n",
       " 'ever': 335,\n",
       " '09064019788': 4811,\n",
       " 'dramatic': 4812,\n",
       " 'sigh': 3023,\n",
       " 'cttargg': 4813,\n",
       " 'flights': 1415,\n",
       " 'lyrics': 4814,\n",
       " 'keepintouch': 7144,\n",
       " 'motorola': 823,\n",
       " 'comedy': 2278,\n",
       " '2u': 1416,\n",
       " 'smiles': 1616,\n",
       " '3lp': 3610,\n",
       " 'lanre': 6910,\n",
       " 'death': 2279,\n",
       " 'manda': 1919,\n",
       " 'mineall': 4822,\n",
       " 'realy': 1624,\n",
       " 'he': 71,\n",
       " '46': 6249,\n",
       " 'buses': 3028,\n",
       " 'sorry': 83,\n",
       " 'lots': 649,\n",
       " 'they': 105,\n",
       " 'i\\x92llspeak': 8842,\n",
       " 'shanghai': 7148,\n",
       " 'glorious': 4824,\n",
       " 'recharge': 7899,\n",
       " 'sry': 3031,\n",
       " '09058099801': 3032,\n",
       " 'scarcasim': 4825,\n",
       " 'omw': 1257,\n",
       " 'gota': 7454,\n",
       " 'wmlid1b6a5ecef91ff937819firsttrue180430jul05': 4827,\n",
       " 'denying': 4828,\n",
       " 'happened': 682,\n",
       " 'guess': 322,\n",
       " 'massages': 8473,\n",
       " 'closingdate040902': 4829,\n",
       " 'maturity': 3033,\n",
       " 'jade': 4830,\n",
       " 'kuch': 4831,\n",
       " 'sent': 185,\n",
       " 'w111wx': 1882,\n",
       " 'strike': 2280,\n",
       " 'ex': 963,\n",
       " 'deltomorrow': 4832,\n",
       " 'waitu': 2281,\n",
       " 'frwd': 4833,\n",
       " 'oz': 1617,\n",
       " 'meetin': 2168,\n",
       " 'booked': 964,\n",
       " 'format': 3034,\n",
       " 'mis': 4289,\n",
       " 'heroi': 5930,\n",
       " 'rubber': 4838,\n",
       " 'cashto': 3035,\n",
       " 'pobox11414tcrw1': 4839,\n",
       " 'evrey': 3036,\n",
       " 'memory': 4840,\n",
       " 'short': 1048,\n",
       " 'icic': 4841,\n",
       " 'messed': 4842,\n",
       " 'clover': 4843,\n",
       " 'smile': 273,\n",
       " 'maneesha': 2548,\n",
       " 'sayy': 4844,\n",
       " 'bus': 412,\n",
       " '09066380611': 3037,\n",
       " 'vivek': 4845,\n",
       " 'dear1': 2282,\n",
       " 'sack': 4846,\n",
       " 'record': 2283,\n",
       " 'weighed': 5746,\n",
       " 'seekers': 4847,\n",
       " 'administrator': 3038,\n",
       " 'no': 38,\n",
       " 'any': 102,\n",
       " 'energy': 965,\n",
       " 'lingo': 4848,\n",
       " 'making': 507,\n",
       " 'deliveredtomorrow': 2817,\n",
       " 'yay': 1417,\n",
       " 'nbme': 4849,\n",
       " 'canada': 3039,\n",
       " 'pendingi': 4850,\n",
       " 'buff': 2543,\n",
       " 'friends': 218,\n",
       " 'woot': 3632,\n",
       " 'delete': 3040,\n",
       " 'ultimately': 4852,\n",
       " 'timehope': 4853,\n",
       " 'hiya': 1049,\n",
       " 'woman': 1618,\n",
       " 'fringe': 3041,\n",
       " 'greet': 2285,\n",
       " '08718738002': 7199,\n",
       " 'mgs': 7619,\n",
       " 'shipped': 3042,\n",
       " '2007': 3043,\n",
       " '09041940223': 4858,\n",
       " 'macedonia': 8852,\n",
       " 'hum': 4860,\n",
       " 'contents': 2286,\n",
       " 'chinatown': 4862,\n",
       " 'lodging': 4863,\n",
       " 'guessing': 3044,\n",
       " 'sign': 1150,\n",
       " 'engin': 3045,\n",
       " 'patty': 4864,\n",
       " 'buyers': 4865,\n",
       " 'operator': 716,\n",
       " 'shuhui': 1328,\n",
       " 'chances': 3046,\n",
       " 'facts': 9609,\n",
       " 'actor': 3047,\n",
       " 'gsoh': 4867,\n",
       " 'janinexx': 4868,\n",
       " 'railway': 2287,\n",
       " 'admin': 3048,\n",
       " 'gain': 3563,\n",
       " 'text82228': 4870,\n",
       " 'kano': 2288,\n",
       " 'day2': 6004,\n",
       " 'debating': 4871,\n",
       " 'bbdeluxe': 4872,\n",
       " '10am7pm': 1884,\n",
       " 'wrk': 3049,\n",
       " '87021': 2289,\n",
       " 'dawhats': 3674,\n",
       " 'recovery': 3050,\n",
       " 'dis': 400,\n",
       " 'sells': 3051,\n",
       " 'bothering': 4874,\n",
       " 'scratches': 4875,\n",
       " 'improved': 3052,\n",
       " '150p': 516,\n",
       " 'jos': 8857,\n",
       " 'rsi': 4878,\n",
       " 'onwards': 2678,\n",
       " 'keyword': 4881,\n",
       " 'xnet': 4882,\n",
       " 'documents': 4883,\n",
       " 'persevered': 7347,\n",
       " 'reservations': 4885,\n",
       " 'violet': 3053,\n",
       " 'garments': 5077,\n",
       " 'dating': 683,\n",
       " 'less': 1019,\n",
       " 'wan2': 3737,\n",
       " 'melike': 6975,\n",
       " 'massagetiepos': 4890,\n",
       " 'traveling': 4891,\n",
       " 'sipix': 1418,\n",
       " 'startedindia': 4892,\n",
       " 'rummer': 4893,\n",
       " 'pataistha': 4894,\n",
       " 'birthdate': 3054,\n",
       " 'nite': 562,\n",
       " 'i': 1,\n",
       " 'resent': 4895,\n",
       " 'careless': 4897,\n",
       " 'sonathaya': 4898,\n",
       " 'claimcode': 4899,\n",
       " 'brum': 4900,\n",
       " 'venugopal': 4901,\n",
       " 'insurance': 1259,\n",
       " 'wereare': 4902,\n",
       " '08712317606': 3056,\n",
       " 'chief': 5464,\n",
       " 'call2optoutj': 7494,\n",
       " 'servs': 4905,\n",
       " 'coccooning': 4906,\n",
       " '945': 4907,\n",
       " 'again': 195,\n",
       " '£1500': 1885,\n",
       " 'shirts': 2755,\n",
       " '88877free': 4909,\n",
       " 'temple': 2077,\n",
       " 'expert': 4911,\n",
       " 'opinions': 3057,\n",
       " 'gbp450week': 4912,\n",
       " '87077': 1151,\n",
       " 'swalpa': 4913,\n",
       " 'ummmmmaah': 2290,\n",
       " 'rons': 4914,\n",
       " 'terrific': 3762,\n",
       " '2im': 4915,\n",
       " 'ridden': 4916,\n",
       " 's': 401,\n",
       " 'declare': 5295,\n",
       " 'approx': 3059,\n",
       " 'wwwfullonsmscom': 3060,\n",
       " 'givits': 4918,\n",
       " 'without': 428,\n",
       " 'ee': 3061,\n",
       " 'friendshipmotherfatherteacherschildrens': 4919,\n",
       " '08448714184': 4920,\n",
       " 'howre': 4921,\n",
       " 'polo': 3369,\n",
       " 'lush': 3063,\n",
       " 'cancelled': 3775,\n",
       " 'tour': 1419,\n",
       " 'ls1': 3064,\n",
       " 'loxahatchee': 3065,\n",
       " 'lonely': 2109,\n",
       " 'uup': 4923,\n",
       " 'unsold': 2679,\n",
       " 'cheek': 4888,\n",
       " 'chores': 4926,\n",
       " 'hee': 800,\n",
       " 'kappa': 3066,\n",
       " 'woods': 4927,\n",
       " 'contented': 4928,\n",
       " 'dengra': 4929,\n",
       " 'itna': 4930,\n",
       " 'has': 111,\n",
       " 'put': 413,\n",
       " 'puzzles': 4931,\n",
       " 'brats': 4932,\n",
       " '116': 4933,\n",
       " 'okcome': 4934,\n",
       " 'oil': 3067,\n",
       " 'early': 355,\n",
       " 'software': 2292,\n",
       " 'canlove': 4935,\n",
       " 'growing': 3068,\n",
       " 'hill': 1886,\n",
       " 'cares': 2293,\n",
       " 'kg': 3804,\n",
       " 'knock': 4331,\n",
       " 'imagine': 1260,\n",
       " 'uncut': 7799,\n",
       " 'twelve': 2680,\n",
       " 'social': 1619,\n",
       " 'ayn': 3070,\n",
       " 'moan': 1420,\n",
       " '£150perwksub': 3071,\n",
       " 'hard': 684,\n",
       " '80488biz': 4940,\n",
       " '£1minmobsmorelkpobox177hp51fl': 4941,\n",
       " 'starting': 1050,\n",
       " 'mack': 4944,\n",
       " 'web': 3072,\n",
       " 'rayman': 4946,\n",
       " 'somewhere': 967,\n",
       " 'cuddling': 3073,\n",
       " 'holby': 4947,\n",
       " 'near': 968,\n",
       " 'corporation': 9122,\n",
       " 'slowing': 4948,\n",
       " 'fring': 4949,\n",
       " 'revision': 3074,\n",
       " 'cu': 4950,\n",
       " 'burger': 2294,\n",
       " 'ciao': 4783,\n",
       " 'voda': 1620,\n",
       " 'walkin': 7063,\n",
       " 'write': 969,\n",
       " 'cozy': 8496,\n",
       " 'nightswt': 4953,\n",
       " 'curtsey': 6275,\n",
       " 'jamstercouk': 7175,\n",
       " 'refused': 1887,\n",
       " 'trishul': 8019,\n",
       " 'sugar': 1888,\n",
       " 'used': 726,\n",
       " 'miiiiiiissssssssss': 4958,\n",
       " 'ts': 2295,\n",
       " 'way2smscom': 9293,\n",
       " 'oja': 4960,\n",
       " 'glad': 970,\n",
       " 'slave': 1152,\n",
       " ...}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab_to_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Convert the reviews to integers, same shape as reviews list, but with integers\n",
    "text_ints = []\n",
    "for each in texts:\n",
    "    text_ints.append([vocab_to_int[word] for word in each.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[5, 234, 140, 23, 355, 3900, 5, 160, 144, 59, 140]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_ints[3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zero-length text: 2\n",
      "Maximum text length: 171\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "text_lens = Counter([len(x) for x in text_ints])\n",
    "print(\"Zero-length text: {}\".format(text_lens[0]))\n",
    "print(\"Maximum text length: {}\".format(max(text_lens)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5572"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "non_zero_idx = [ii for ii, texts in enumerate(text_ints) if len(texts) != 0]\n",
    "len(non_zero_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5574"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# fillter out that review with 0 length\n",
    "text_ints = [text_ints[ii] for  ii in non_zero_idx]\n",
    "labels = np.array([labels[ii] for ii in non_zero_idx])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "create an array features that contains the data we'll pass to the network. Each row should be 170 elements long. For text shorter than 170 words, left pad with 0s. For text longer than 170, use on the first 170 words as the feature vector."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seq_len = 170\n",
    "features = np.zeros((len(text_ints), seq_len), dtype=int)\n",
    "for i, row in enumerate(text_ints):\n",
    "    features[i, -len(row):] = np.array(row)[:seq_len]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,   45,  449, 6432,  839,\n",
       "        740,  701,   63,    8, 1380,   89,  120,  357, 1332,  153, 3704,\n",
       "       1367,   67,   56, 8992,  137])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training, Validation, Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tFeature Shapes:\n",
      "Train set: \t\t(4457, 170) \n",
      "Validation set: \t(557, 170) \n",
      "Test set: \t\t(558, 170)\n"
     ]
    }
   ],
   "source": [
    "split_frac = 0.8\n",
    "split_idx = int(len(features)*0.8)\n",
    "train_x, val_x = features[:split_idx], features[split_idx:]\n",
    "train_y, val_y = labels[:split_idx], labels[split_idx:]\n",
    "\n",
    "test_idx = int(len(val_x)*0.5)\n",
    "val_x, test_x = val_x[:test_idx], val_x[test_idx:]\n",
    "val_y, test_y = val_y[:test_idx], val_y[test_idx:]\n",
    "\n",
    "print(\"\\t\\t\\tFeature Shapes:\")\n",
    "print(\"Train set: \\t\\t{}\".format(train_x.shape), \n",
    "      \"\\nValidation set: \\t{}\".format(val_x.shape),\n",
    "      \"\\nTest set: \\t\\t{}\".format(test_x.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lstm_size = 256\n",
    "lstm_layers = 2\n",
    "batch_size = 250\n",
    "learning_rate = 0.001\n",
    "drop_out = 0.5\n",
    "epochs = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create TF Placeholders for the Neural Network."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_words = len(vocab_to_int)\n",
    "\n",
    "# Create the graph object\n",
    "graph = tf.Graph()\n",
    "# Add nodes to the graph\n",
    "with graph.as_default():\n",
    "    inputs_ = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "    labels_ = tf.placeholder(tf.int32, [None, None], name='labels')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Embedding\n",
    "There are about 1000 words in our vocabulary. It is massively inefficient to one-hot encode. Instead of one-hot encoding, we can have an embedding layer and use that layer as a lookup table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Size of the embedding vectors (number of units in the embedding layer)\n",
    "embed_size = 300 \n",
    "\n",
    "with graph.as_default():\n",
    "    embedding = tf.Variable(tf.random_uniform((n_words, embed_size), -1, 1))\n",
    "    embed = tf.nn.embedding_lookup(embedding, inputs_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build RNN Cell and Initialize\n",
    "Stack one or more LSTMCells in a MultiRNNCell. (if you are using tensflow < 1.0, you may get errors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    def lstm_cell():\n",
    "        cell = tf.contrib.rnn.LSTMCell(lstm_size, \n",
    "                                       initializer=tf.random_uniform_initializer(-0.1, 0.1, seed=2),\n",
    "                                       state_is_tuple=True)\n",
    "        drop = tf.contrib.rnn.DropoutWrapper(cell, output_keep_prob=keep_prob)\n",
    "        return drop\n",
    "    \n",
    "    stack_cells = tf.contrib.rnn.MultiRNNCell([lstm_cell() for _ in range(lstm_layers)])\n",
    "    \n",
    "    initial_state = state = stack_cells.zero_state(batch_size, tf.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    outputs, final_state = tf.nn.dynamic_rnn(stack_cells, embed, initial_state=initial_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    predictions = tf.contrib.layers.fully_connected(outputs[:, -1], 1, activation_fn=tf.sigmoid)\n",
    "    cost = tf.losses.mean_squared_error(labels_, predictions)\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    correct_pred = tf.equal(tf.cast(tf.round(predictions), tf.int32), labels_)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batches(x, y, batch_size=100):\n",
    "    \n",
    "    n_batches = len(x)//batch_size\n",
    "    x, y = x[:n_batches*batch_size], y[:n_batches*batch_size]\n",
    "    for ii in range(0, len(x), batch_size):\n",
    "        yield x[ii:ii+batch_size], y[ii:ii+batch_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/5 Iteration: 5 Train loss: 0.089\n",
      "Epoch: 0/5 Iteration: 10 Train loss: 0.070\n",
      "Epoch: 0/5 Iteration: 15 Train loss: 0.044\n",
      "Epoch: 1/5 Iteration: 20 Train loss: 0.044\n",
      "Epoch: 1/5 Iteration: 25 Train loss: 0.016\n",
      "Val acc: 0.956\n",
      "Epoch: 1/5 Iteration: 30 Train loss: 0.020\n",
      "Epoch: 2/5 Iteration: 35 Train loss: 0.018\n",
      "Epoch: 2/5 Iteration: 40 Train loss: 0.013\n",
      "Epoch: 2/5 Iteration: 45 Train loss: 0.016\n",
      "Epoch: 2/5 Iteration: 50 Train loss: 0.013\n",
      "Val acc: 0.972\n",
      "Epoch: 3/5 Iteration: 55 Train loss: 0.017\n",
      "Epoch: 3/5 Iteration: 60 Train loss: 0.008\n",
      "Epoch: 3/5 Iteration: 65 Train loss: 0.017\n",
      "Epoch: 4/5 Iteration: 70 Train loss: 0.002\n",
      "Epoch: 4/5 Iteration: 75 Train loss: 0.004\n",
      "Val acc: 0.976\n",
      "Epoch: 4/5 Iteration: 80 Train loss: 0.016\n",
      "Epoch: 4/5 Iteration: 85 Train loss: 0.006\n"
     ]
    }
   ],
   "source": [
    "with graph.as_default():\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    iteration = 1\n",
    "    for e in range(epochs):\n",
    "        state = sess.run(initial_state)\n",
    "        \n",
    "        for ii, (x, y) in enumerate(get_batches(train_x, train_y, batch_size), 1):\n",
    "            feed = {inputs_: x,\n",
    "                    labels_: y[:, None],\n",
    "                    keep_prob: drop_out,\n",
    "                    initial_state: state}\n",
    "            loss, state, _ = sess.run([cost, final_state, optimizer], feed_dict=feed)\n",
    "            \n",
    "            if iteration%5==0:\n",
    "                print(\"Epoch: {}/{}\".format(e, epochs),\n",
    "                      \"Iteration: {}\".format(iteration),\n",
    "                      \"Train loss: {:.3f}\".format(loss))\n",
    "\n",
    "            if iteration%25==0:\n",
    "                val_acc = []\n",
    "                val_state = sess.run(stack_cells.zero_state(batch_size, tf.float32))\n",
    "                for x, y in get_batches(val_x, val_y, batch_size):\n",
    "                    feed = {inputs_: x,\n",
    "                            labels_: y[:, None],\n",
    "                            keep_prob: 1,\n",
    "                            initial_state: val_state}\n",
    "                    batch_acc, val_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "                    val_acc.append(batch_acc)\n",
    "                print(\"Val acc: {:.3f}\".format(np.mean(val_acc)))\n",
    "            iteration +=1\n",
    "    saver.save(sess, \"checkpoints/sentiment.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoints/sentiment.ckpt\n",
      "Test accuracy: 0.988\n"
     ]
    }
   ],
   "source": [
    "test_acc = []\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n",
    "    test_state = sess.run(stack_cells.zero_state(batch_size, tf.float32))\n",
    "    for ii, (x, y) in enumerate(get_batches(test_x, test_y, batch_size), 1):\n",
    "        feed = {inputs_: x,\n",
    "                labels_: y[:, None],\n",
    "                keep_prob: 1,\n",
    "                initial_state: test_state}\n",
    "        batch_acc, test_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "        test_acc.append(batch_acc)\n",
    "    print(\"Test accuracy: {:.3f}\".format(np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
